// rs_convert.cpp: conversion between different orthographies
#define _CRT_SECURE_NO_DEPRECATE 1
#include <string>
#include <iostream>
#include <fstream>
#include <iomanip>
#include <sstream>
#include <boost/filesystem.hpp>
#include <process.h>
namespace fs = boost::filesystem;
#include "rs_table.h"
#include "rs_string.h"
#include "rs_stream.h"
#include "rs_log.h"
#include "rs_token.h"
#include "rs_convert.h"

/*

formats:
instant messaging
html
word processor & text editor
software localization
pdf

// todo dictionaries have data with ' and -, use them
 
// todo: when converting a text body, get proposals which words to add to dictionary
// when many words could be stemmed to the same base word
// similarly for prefixes like aero- that are not words on their own but are frequent

words not in dictionary, by similarity
incorrectly spelled words
words that have more that one possible output form
  selection of format (british, american; specific wanted pronuciation)
foreign words, proper names, other unchangeables
words with accents - letters other than usual 2*26 characters


// how to detect misspelt words?
// add words to the dictionary when once stemmed
// check dictionary with first try to stem
// add word measure m: [C](VC){m}[V] as condition for affixes

check only once for inflectional postfixes as in RO 'doest'
to          ro          to_base     ro_base
dosed       doest       dose        do   

mark all words with inflectional postfixes as not applicable to be base of words with derivational postfixes
foresee     forsee      fores       fors 

correct stemming rider-> "ride" and not "rid"

mark words like "sister" as without affixes and not having -er

after word is split, try to compose it in the same rs_table and check the result if it is the same

AF fierhouzes doesn't split
HOUSE  HH AW1 S
FIREHOUSE  F AY1 ER0 HH AW2 S
FIREHOUSES  F AY1 ER0 HH AW2 S IH0 Z
 14901 firehouse        fierhous        
 14902 firehouses       fierhouzes      
 18980 house            houz            
 18981 house            hous            

*/

converter::converter (
    rs_grammar* rsg,
    const rs_string& src_og_name,
    const rs_string& tgt_og_name,
    bool show_unknown,
    bool allow_same)
{ 
    cnv_rsg=rsg;
    cnv_dir = false;
    cnv_stat.cnv_missing.add_text_col (L"unknown_words", true, 20);
    cnv_ifs=0;
    cnv_ofs=0;
    cnv_stat.cnv_show_unknown=show_unknown;
    cnv_report_stat= true;
    cnv_valid = rsg->is_valid();
    if (!cnv_valid)
        return;
    cnv_valid = cnv_rsg->init_og_pair (cnv_rsp, src_og_name, tgt_og_name, allow_same);
}



bool converter::convert_any_word (const rs_string& src_word, rs_string& tgt_word)
{ // converts any token from source orthography to target orthography
    // handles upper-case words and abbreviations
    tgt_word = src_word;
    if (!is_any_alpha(src_word) || src_word.length()==0) {
        return true;
    }
    rs_token token (src_word, true);
    token.normalize();
    if (token.has_nonascii()) {
        return false;
    }
    word_case wcase = token.get_case();
    if (wcase == wc_upper || wcase == wc_mixed) {
        return true; // no change for abbreviations and mixed casing
    }
    // todo pass norm_data to convert_normalized_word to e.g.
    // todo (cont) leave capitalized src_word as they are (Proper names?) if not in dictionary
    rs_string norm_word = token.get_string(), norm_tgt;
    bool result = cnv_rsg->convert_normalized_word (norm_word, norm_tgt, true, 
        true, cnv_stat, cnv_rsp);
    // todo: either pass token to convert_normalized_word or get rid of token in this function
    if (result ||  wcase != wc_init_upper) {
        token.set_string (norm_tgt);
        token.denormalize ();
        tgt_word = token.get_string();
    }
    return result;
    // todo allow general case words in dictionary if special flag is set
}



bool converter::convert_article (const wstring& src, wstring& tgt, const wd_mode& wdm)
{
    // converts an article 'src' to 'tgt'
    // optionally rearranges tokens like hyphenated words
    // todo impove homograph disambiguation verb vs. noun: desert, console, refuse, project, ... or some other cases
    // todo add URL detection and supress changing URLs
    tokenized_text tok_text (src, wdm);
    rs_string src_word, tgt_word;
    tgt.clear();
    bool result=true;
    while (!tok_text.eof()) {
        const rs_token& token = tok_text.get_token();
        if (token.get_string().length() == 0)
            break; // end of tokens
        if (!token.is_word()) {
            // non-words are copied only
            tgt += token;
            continue;
        }
        src_word = token;
        const rs_token* token1 = &tok_text.peek_token (0);
        const rs_token* token2 = &tok_text.peek_token (1);
        rs_string insert_str;
        unsigned insert_pos = 0;
        if (wdm == wdm_strings  &&  token1->get_string() == L"&amp;" && token2->is_word()) {
            // for converting .strings file, string like L"P&amp;ercent",
            // mean word L"Percent" with embedded L"&amp;"
            src_word = token.get_string() + token2->get_string();
            insert_str = token1->get_string();
            insert_pos = unsigned (token.get_string().length());
            tok_text.get_token();
            tok_text.get_token();
            token1 = &tok_text.peek_token (0);
            token2 = &tok_text.peek_token (1);
        }
        bool converted(false);
        if (token1->get_string() == L"-" || token1->get_string() == L"'") {
            // let's check if these tokens together can be converted
            src_word = token.get_string() + token1->get_string() + token2->get_string();
            converted = convert_any_word (src_word, tgt_word);
            if (converted) {
                tok_text.get_token();
                tok_text.get_token();
            } else
                src_word = token;
        }
        if (!converted) {
            converted = convert_any_word (src_word, tgt_word);
        }
        if (insert_pos >0) {
            if (insert_pos > tgt_word.length())
                insert_pos = unsigned (tgt_word.length());
            //
            tgt_word.insert (insert_pos, insert_str);
        }
        result = result && converted;
        tgt += tgt_word;
    }
    return result;
}


void converter::update_out_fname (
    const rs_string& in_fname,
    rs_string& out_fname)
{ // if out_fname is not yes specified i.e. still empty,
    // derive it from other data
    if (out_fname!=L"")
        return;
    out_fname = in_fname;
    rs_string replacement;
    replacement = cnv_rsp.rsp_tgt_name;
    if (out_fname.find (L".") != string::npos) {
        replacement=L"-"+replacement+L".";
        replace_file_substring (out_fname, L".", replacement.c_str());
    } else {
        out_fname += L"-" + replacement;
    }
}

bool converter::init_open_files (const rs_string& in_fname, rs_string& out_fname, bool skip_input_output)
{   // creates and opens input stream
    // creates output stream
    // initializes statistic counters
    if (cnv_report_stat) {
        cnv_stat.init_counters();
    }
    update_out_fname (in_fname, out_fname);
    if (skip_input_output) {
        return true;
    }
    if (cnv_ifs!=0  ||  cnv_ofs!=0) {
        cout << "error: conversion streams are already open" << endl;
        return false;
    }
    cnv_ifs = new rs_istream (in_fname);
    rs_encoding enc = cnv_ifs->get_encoding();
    cnv_ofs = new rs_ostream (out_fname, enc);
    cnv_ofname = out_fname;
    if (!cnv_ifs->good()  ||  !cnv_ofs->is_open()) {
        return false;
    }
    return true;
}


bool converter::close_files ()
{
    if (cnv_ifs==0 || cnv_ofs==0) {
        cout << "error: cnv_ifs and cnv_ofs, at least one not present" << endl;
        return false;
    }
    delete cnv_ifs;
    cnv_ifs=0;
    delete cnv_ofs;
    cnv_ofs=0;
    return true;
}


void converter::report_statistics()
{
    if (!cnv_report_stat)
        return;
    rs_log_console (L"Conversion finished, words: "
        + rs_stringed(cnv_stat.counter (rse_dict)) + L" in dict, "
        + rs_stringed(cnv_stat.counter (rse_stemmed)) + L" stemmed, "
        + rs_stringed(cnv_stat.counter (rse_unknown)) + L" unknown");

    rs_log_file (L"Unknown words");
    for (rs_rix rix=rs_rix::begin(); rix<cnv_stat.cnv_missing.get_row_count(); rix++) {
        const rs_string& word (cnv_stat.cnv_missing.get_text (text_cix(0), rix));
        rs_log_file (word);
    }
}

void converter::convert_po_file ()
{
    wstring line, cnv_line;
    vector<wstring> src, cnv; // source and translated lines
    enum po_mode {po_none, po_src, po_cnv};

    po_mode pm=po_none;
    size_t lines=0;
    while (!cnv_ifs->eof()) {
        cnv_ifs->read_line (line);
        lines++;
        wostringstream woss;
        if (line.length() == 0) {
            woss << line;
            pm=po_none;
        } else if (line[0] == L'#') {
            woss << line;
        } else if (line.substr (0, 6).compare (L"msgid ") == 0) {
            src.clear();
            line = line.substr (6);
            pm = po_src;
            woss << L"msgid ";
        } else if (line.substr (0, 7).compare (L"msgstr ") == 0) {
            cnv.clear();
            for (rs_string_v_it it=src.begin(); it!=src.end(); it++) {
                convert_article (*it, cnv_line, wdm_po);
                cnv.push_back (cnv_line);
            }
            woss << L"msgstr ";
            for (rs_string_v_it it=cnv.begin(); it!=cnv.end(); it++)
                woss << *it << std::endl;
            pm = po_cnv;
        } else if (line[0] == '"') {
            // nothing to do here
        } else
            rs_log_file (L"??? what's this " + line);
        if (pm == po_src) {
            src.push_back (line);
            woss << line << endl;
        }
    }
}


void converter::convert_strings_file ()
{   // converts file with extension .strings
    // used for localization of program Abiword 2.6
    bool conversion_recorded=false;

    wstring line, cnv_line;
    wstring src, cnv;
    size_t lines=0;
    size_t equ_pos=0;
    while (!cnv_ifs->eof()) {
        cnv_ifs->read_line (line);
        lines++;
        if (line.length() == 0) {
            if (!conversion_recorded) {
                conversion_recorded = true;
                wstring signature = L"<!-- Generated by program respell -->";
                cnv_ofs->write_line (signature);
            } else 
                cnv_ofs->write_line (line);
        } else if (line[0] == L'<'  ||  line[0] == L'/') {
            cnv_ofs->write_line (line);
        } else if ((equ_pos=line.find (L"=")) != string::npos) {
            src = line.substr (equ_pos+1);
            convert_article (src, cnv, wdm_strings);
            cnv_line = line.substr (0, equ_pos+1) + cnv;
            cnv_ofs->write_line (cnv_line);
            rs_log_file (L"TO: " + line);
            rs_log_file (cnv_rsp.rsp_tgt_name+L": " + cnv_line);
        } else
            rs_log_file (L"error: unknown meaning of line: " + line);
    }
}


void converter::convert_text_file ()
{
    // converts text file
    wstring line, conv;
    int line_no=0;
    while (!cnv_ifs->eof()) {
        line_no++;
        cnv_ifs->read_line (line, true);
        convert_article (line, conv, wdm_qd);
        cnv_ofs->write_line (conv);
        rs_log_file (L"TO: " + line);
        rs_log_file (cnv_rsp.rsp_tgt_name+L": " + conv);
    }
}


void converter::convert_ispell_file ()
{
    // converts .ispell dictionary file: for each line converts text before slash (/) or end-of-line
/* ispell for Abiword 2.6 for windows
http://www.crazy-wormhole.com/AlinaMeridon/AbiWord/abidocs/info/infospelling.html
AbiWord expects its Ispell dictionaries to be in the following format: 
128 byte long strings, 52 "flags", capitalization enabled, little endian 
checking:
/usr/lib/ispell/norsk.hash: little endian ispell 3.1 hash file, 8-bit, capitalization, 52 flags and 128 string characters
ispell for windows: 
http://www.fsci.fuk.kindai.ac.jp/kakuto/win32-ptex/web2c75-e.html
ftp://fsci.fuk.kindai.ac.jp/pub/ptex/utils/ispell-3.3.02-w32.tar.gz
ftp://fsci.fuk.kindai.ac.jp/pub/ptex/utils/ispell-3.3.02-w32-src.tar.bz2
dictionaries at ftp://ftp.tue.nl/pub/tex/GB95/ispell-english.zip.
abi-orig:
american.hash: little endian ispell 3.1 hash file, 7-bit, capitalization, 52 flags and 128 string characters
kakuto-w32:
uk_med.hash: little endian ispell 3.1 hash file, 7-bit, capitalization, 52 flags and 512 string characters
*/
    wstring line, conv;
    int line_no=0;
    cnv_stat.cnv_show_unknown = false;
    while (!cnv_ifs->eof()) {
        line_no++;
        cnv_ifs->read_line (line);
        vector<rs_string> line_v;
        string_to_vec (line, line_v, L"/");
        if (line_v.size() > 0) {
            convert_article (line_v[0], conv, wdm_qd);
            if (cnv_keep_to && line_v[0] != conv) {
                // keep also TO word
                cnv_ofs->write_line (line);
            }
            line_v[0] = conv;
            vec_to_string (line_v, conv, L"/");
            cnv_ofs->write_line (conv);
            rs_log_file (L"TO: " + line);
            rs_log_file (cnv_rsp.rsp_tgt_name+L": " + conv);
        } else {
            cnv_ofs->write_line (line);
        }
    }
    cnv_ofs->close();

    rs_string ispell_aff_fname (L"english.aff");
    if (!file_exists (ispell_aff_fname)) {
        rs_report_error  (L"Missing ispell affix file: ", ispell_aff_fname);
        return;
    }
    rs_string hash_fname (cnv_ofname);
    hash_fname += L".hash";
    replace_file_substring (hash_fname, L".ispell.hash", L".hash");
    create_ispell_hash (cnv_ofname, ispell_aff_fname, hash_fname);
}


void converter::create_ispell_hash (const rs_string& dict_fname, 
                         const rs_string& ispell_aff_fname,
                         const rs_string& hash_fname)
{
    if (!cnv_dir) {
        rs_log_console (L"Creating ispell dictionary: "+hash_fname +
            L" from files " +dict_fname + L" and " + ispell_aff_fname);
    }
    rs_string cmd = L"buildhash.exe -s " + dict_fname + L" " + ispell_aff_fname
        + L" " + hash_fname + L" >> buildhash.log 2>&1";
    _wsystem (cmd.c_str());
}


void converter::convert_html_file ()
{ // converts html file
    // todo: changes visible text TO<->RO
    // todo: changes "lang" attributes
    // todo: changes links (if specified by argument)
    int line_no=0;
    const int max_html_size=10000000;
    int64 fsize64=cnv_ifs->file_size();
    if (fsize64 >= max_html_size) {
        cout << "error: input file too long " << fsize64 
            << " bytes, max possible size is " << max_html_size << endl;
        return;
    }
    wstring ihtm;
    cnv_ifs->read_file (ihtm);
    size_t isize=ihtm.length();
    const wchar_t* ichtm=ihtm.c_str();
    size_t cpos=0; // current position
    size_t pos;
    wstring piece, conv;
    while (cpos < isize) {
        // searching for '<'
        pos = cpos;
        while (pos < isize  && ichtm[pos] != L'<')
            pos++;
        // range cpos..pos can be translated
        if (pos>cpos) {
            piece = ihtm.substr (cpos, pos-cpos);
            convert_article (piece, conv, wdm_qd);
            cnv_ofs->write_line (conv);
            rs_log_file (L"TO: " + piece);
            rs_log_file (cnv_rsp.rsp_tgt_name+L": " + conv);
        }
        cpos = pos;
        while (pos < isize  && ichtm[pos] != L'>')
            pos++;
        if (pos < isize)
            pos++;
        // range cpos..pos will be copied unchanged
        if (pos>cpos) {
            piece = ihtm.substr (cpos, pos-cpos);
            cnv_ofs->write_line (piece);
        }
        cpos = pos;
    }
}


bool converter::convert_file (const rs_string& in_fname,
                              rs_string& out_fname,
                              const rs_string& format,
                              bool keep_to)
{
    if (!cnv_valid) {
        rs_report_error (L"no valid grammar");
        return false;
    }
    cnv_keep_to = keep_to;
    rs_string ext (format);
    if (format == L"") {
        // autodetection
        size_t dot_pos = in_fname.rfind (L".");
        if (dot_pos != string::npos)
            ext = in_fname.substr (dot_pos);
    }
    to_lower (ext);
    if (ext.substr(0,1) == L".")
        ext = ext.substr (1);
    file_format ff;
    if (ext == L"txt")
        ff = ff_text;
    else if (ext==L"ispell")
        ff = ff_ispell;
    else if (ext==L"po")
        ff = ff_po;
    else if (ext==L"strings")
        ff = ff_strings;
    else if (ext==L"html" || ext==L"htm")
        ff = ff_html;
    else
        ff = ff_default;
    if (ff != ff_default) {
        if (!init_open_files (in_fname, out_fname))
            return false;
    }
    if (!cnv_dir)
        rs_log_console (L"Converting file " + in_fname + L" to file " + out_fname);
    switch (ff) {
        case ff_text:
            convert_text_file ();
            break;
        case ff_ispell:
            convert_ispell_file ();
            break;
        case ff_po:
            convert_po_file ();
            break;
        case ff_strings:
            convert_strings_file ();
            break;
        case ff_html:
            convert_html_file ();
            break;
        default:
            if (cnv_dir) {
                //cout << "    Copying it" << endl; // << in_fname << " to " << out_fname << endl;
                fs::copy_file (in_fname, out_fname);
            } else {
                rs_report_error (L"no known conversion");
                return false;
            }
            break;
    }
    if (ff != ff_default) {
        close_files ();
        report_statistics();
    }
    return true;
}


bool converter::convert_directory (const rs_string& in_dir,
                                   const rs_string& out_dir_base,
                                   bool keep_to)
{
    cnv_keep_to = keep_to;
    return convert_directory (in_dir, out_dir_base, 0);
}

bool converter::convert_directory (const rs_string& in_dir, const rs_string& out_dir_base, int level)
{   // converts all files from one directory to another
    // files with unknown extensions are copied unchanged
    // recursive function
    if (!cnv_valid) {
        rs_report_error (L"no valid grammar");
        return false;
    }
    cnv_dir = true;
    fs::path full_path( fs::initial_path<fs::path>() );
    full_path = fs::system_complete (fs::path(stringed(in_dir)));
    if (!fs::exists(full_path)) {
        rs_report_error (L"input directory not found: ", in_dir);
        return false;
    }
    if (!fs::is_directory( full_path)) {
        rs_report_error (L"input directory is actually not directory: ", in_dir);
        return false;
    }
    rs_string out_dir (out_dir_base);
    update_out_fname (in_dir, out_dir);
    if (level == 0)
        rs_log_console (L"Converting directory " + in_dir + L" to " + out_dir);
    if (level==0) {
        if (!init_open_files (in_dir, out_dir, true))
            return false;
    }
    cnv_report_stat=false;    
    if (!fs::exists(stringed(out_dir))) 
    {
        fs::create_directory (out_dir);
    }
    else 
    {
        //rs_report_error (L"output directory already exists: ", out_dir);
        //return false;
    }
    fs::directory_iterator end_iter;
    for ( fs::directory_iterator dir_itr( full_path );
        dir_itr != end_iter;
        ++dir_itr )
    {
        try
        {
            if (fs::is_directory( dir_itr->status())) {
                convert_directory (in_dir + L"\\" + rs_stringed(dir_itr->path().filename()),
                    out_dir + L"\\" + rs_stringed(dir_itr->path().filename()),
                    level+1);
            }
            else if (fs::is_regular_file( dir_itr->status())) {
                convert_file (in_dir + L"\\" + rs_stringed(dir_itr->path().filename()),
                    out_dir + L"\\" + rs_stringed(dir_itr->path().filename()),
                    L"",
                    cnv_keep_to);
            }
        }
        catch (const exception& e) {
            cout << dir_itr->path().filename() << " " << e.what() << endl;
        }
    }
    if (level==0) {
        cnv_report_stat = true;
        report_statistics();
        cnv_dir = false;
    }
    return true;
}


converter& converter_factory::get_converter (
    const rs_string& src_og_name,
    const rs_string& tgt_og_name,
    const rs_string& affix_cg_fname,
    const rs_string& proper_lc_fname,
    bool show_unknown,
    bool allow_same)
{
    if (!cf_grammar) {
        cf_grammar = new rs_grammar (affix_cg_fname, proper_lc_fname, L"SP"); // todo: either way
    }
    converter* cnv = new converter (cf_grammar, src_og_name, tgt_og_name, show_unknown, allow_same);
    return *cnv;
}
